{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "helper load finish!!!\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from base_helper import *\n",
    "test_op = get_operation_round1_new()\n",
    "train_op_tr = get_operation_train_new()\n",
    "train_tst = get_transaction_train_new()\n",
    "test_tst = get_transaction_round1_new()\n",
    "\n",
    "tag = get_tag_train_new()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 两张表中相同字段，整体以及不同的字段的样本标签的分布情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The train label uid number is: 31179\n",
      "The train data operation table uid is:29728\n",
      "The train data transction table uid is:30542\n",
      "The train all uid is :31179\n",
      "The number 29091 not in two table\n",
      "The train data ratio:0.9330318483594727\n",
      "The train op ratio:0.9785723896663079, and the train tst ratio:0.9524916508414643\n"
     ]
    }
   ],
   "source": [
    "tag_uid = tag.UID.unique()\n",
    "print(\"The train label uid number is: {}\".format(len(tag_uid)))\n",
    "train_op_uid = train_op_tr.UID.unique()\n",
    "print(\"The train data operation table uid is:{}\".format(len(train_op_uid)))\n",
    "train_tst_uid = train_tst.UID.unique()\n",
    "print(\"The train data transction table uid is:{}\".format(len(train_tst_uid)))\n",
    "train_all_uid = set(train_tst_uid) | set(train_op_uid)\n",
    "print(\"The train all uid is :{}\".format(len(train_all_uid)))\n",
    "train_common_uid = set(train_tst_uid) & set(train_op_uid)\n",
    "print(\"The number {} not in two table\".format(len(set(train_common_uid))))\n",
    "print(\"The train data ratio:{}\".format(len(train_common_uid) / len(tag.UID.unique())))\n",
    "print(\"The train op ratio:{}, and the train tst ratio:{}\".format(len(train_common_uid) / len(train_op_uid), \n",
    "                                                                 len(train_common_uid) / len(train_tst_uid)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The test label uid number is: 31179\n",
      "The test data operation table uid is:27887\n",
      "The test data transction table uid is:30101\n",
      "The number 26790 commom in two table\n",
      "The all test uid is :31198\n",
      "The test op ratio:0.9606626743643992, and the test tst ratio:0.8900036543636424\n"
     ]
    }
   ],
   "source": [
    "print(\"The test label uid number is: {}\".format(len(tag.UID.unique())))\n",
    "test_op_uid = test_op.UID.unique()\n",
    "print(\"The test data operation table uid is:{}\".format(len(test_op_uid)))\n",
    "test_tst_uid = test_tst.UID.unique()\n",
    "print(\"The test data transction table uid is:{}\".format(len(test_tst_uid)))\n",
    "test_common_uid = set(test_tst_uid) & set(test_op_uid)\n",
    "test_all_uid = set(test_tst_uid) | set(test_op_uid)\n",
    "print(\"The number {} commom in two table\".format(len(set(test_common_uid))))\n",
    "# print(\"The test data ratio:{}\".format(len(common_uid) / len(tag.UID.unique())))\n",
    "print(\"The all test uid is :{}\".format(len(set(test_all_uid))))\n",
    "print(\"The test op ratio:{}, and the test tst ratio:{}\".format(len(test_common_uid) / len(test_op_uid), \n",
    "                                                                 len(test_common_uid) / len(test_tst_uid)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_op_merge = pd.merge(train_op_tr, tag, on='UID', how='left')\n",
    "train_tst_merge = pd.merge(train_tst, tag, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29728 0.14387109795479008\n",
      "30542 0.1307380001309672\n",
      "31179 0.13743224606305526\n"
     ]
    }
   ],
   "source": [
    "'''每张表中各自的样本标签的分布情况'''\n",
    "print(len(train_op_uid), len(train_op_merge[train_op_merge.Tag == 1]['UID'].unique()) / len(train_op_uid))\n",
    "print(len(train_tst_uid), len(train_tst_merge[train_tst_merge.Tag == 1]['UID'].unique()) / len(train_tst_uid))\n",
    "print(len(tag_uid), len(tag[tag.Tag == 1]['UID'].unique()) / len(tag_uid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the all uid :31179, and the common uid :29091\n",
      "The no_common number :29091, and the ratio :0.13698394692516586\n"
     ]
    }
   ],
   "source": [
    "'''train中共有的样本的标签的分布情况'''\n",
    "print(\"the all uid :{}, and the common uid :{}\".format(len(train_all_uid), len(train_common_uid)))\n",
    "tag_common = tag[tag.UID.isin(train_common_uid)].reset_index(drop=True)\n",
    "print(\"The no_common number :{}, and the ratio :{}\".format(len(tag_common.UID.unique()), \n",
    "                                    len(tag_common[tag_common.Tag == 1]['UID'].unique()) \n",
    "                                                           / len(tag_common.UID.unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the all uid :31179, and the common uid :29091\n",
      "2088\n",
      "The no_common number :2088, and the ratio :0.14367816091954022\n"
     ]
    }
   ],
   "source": [
    "'''train中不是共有的样本的标签分布情况'''\n",
    "print(\"the all uid :{}, and the common uid :{}\".format(len(train_all_uid), len(train_common_uid)))\n",
    "no_common = list(set(train_all_uid) - set(train_common_uid))\n",
    "print(len(no_common))\n",
    "tag_no_common = tag[tag.UID.isin(no_common)].reset_index(drop=True)\n",
    "print(\"The no_common number :{}, and the ratio :{}\".format(len(tag_no_common.UID.unique()), \n",
    "                                    len(tag_no_common[tag_no_common.Tag == 1]['UID'].unique()) \n",
    "                                                           / len(tag_no_common.UID.unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The operation no common:637, and the transction no common:1451\n",
      "The opration Tag == 1 number:292\n",
      "The operation no common ratio:0.45839874411302983\n",
      "The transction Tag == 1 number:8\n",
      "The transction no common ratio:0.005513439007580979\n"
     ]
    }
   ],
   "source": [
    "'''各自两张表中不同的2088个样本的标签比例'''\n",
    "train_op_no_list = list(set(train_op_uid) - set(train_common_uid))\n",
    "train_tst_no_list = list(set(train_tst_uid) - set(train_common_uid))\n",
    "print(\"The operation no common:{}, and the transction no common:{}\".format(len(train_op_no_list),\n",
    "                                                                           len(train_tst_no_list)))\n",
    "train_op_no_common = tag[tag.UID.isin(train_op_no_list)].reset_index(drop=True)\n",
    "train_tst_no_common = tag[tag.UID.isin(train_tst_no_list)].reset_index(drop=True)\n",
    "print(\"The opration Tag == 1 number:{}\".format(\n",
    "                                    len(train_op_no_common[train_op_no_common.Tag == 1]['UID'].unique())))\n",
    "print(\"The operation no common ratio:{}\".format(len(train_op_no_common[train_op_no_common.Tag == 1]['UID'].unique())\n",
    "                                               / len(train_op_no_common.UID.unique())))\n",
    "print(\"The transction Tag == 1 number:{}\".format(\n",
    "                            len(train_tst_no_common[train_tst_no_common.Tag == 1]['UID'].unique())))\n",
    "print(\"The transction no common ratio:{}\".format(len(train_tst_no_common[train_tst_no_common.Tag == 1]['UID'].unique())\n",
    "                                               / len(train_tst_no_common.UID.unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
